{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Reading and Wrangling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "from sklearn import preprocessing\n",
    "\n",
    "import keras.backend as K\n",
    "from keras.layers.core import Activation\n",
    "from keras.models import Sequential,load_model\n",
    "from keras.layers import Dense, Dropout, LSTM\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from keras.callbacks import TensorBoard\n",
    "\n",
    "\n",
    "\n",
    "datadir2 = 'C:/data_week1/MindTitan/data'\n",
    "###############################################################3\n",
    "def do_data_processing(datadir):\n",
    "    '''utility functions'''\n",
    "    def output_len(data):\n",
    "        return(data.shape[0])\n",
    "    \n",
    "    def num(s):\n",
    "        try:\n",
    "            return int(s)\n",
    "        except ValueError:\n",
    "            return float(s)\n",
    "    \n",
    "   \n",
    "    ########### READING DATA ####################33\n",
    "    direct_train = datadir + '/train.csv'\n",
    "    direct_test = datadir + '/test.csv'\n",
    "    direct_rul = datadir + '/rul.csv'\n",
    "    \n",
    "    rawtrain = pd.read_csv(direct_train,engine='python')\n",
    "    rawtest = pd.read_csv(direct_test,engine='python')\n",
    "    test_left_cycles = pd.read_csv(direct_rul,engine='python') # join this with the test data by dataset_id  and unit_id\n",
    "    \n",
    "    #rawtrain.columns = [x.strip() for x in rawtrain.columns] \n",
    "    \n",
    "    rawtest.rename(columns=lambda x: x.strip())\n",
    "    raw_trainlen = output_len(rawtrain)\n",
    "    raw_testlen = output_len(rawtest)\n",
    "    train = rawtrain.copy()\n",
    "    test = rawtest.copy()\n",
    "    together = [rawtrain,rawtest]\n",
    "    rawfull = pd.concat(together)\n",
    "    raw_full_len = output_len(rawfull)\n",
    "    \n",
    "    trainlengths = [sum(train['dataset_id']=='FD00'+str(i)) for i in range(1,5)]\n",
    "    sum(trainlengths) == raw_trainlen\n",
    "    testlengths = [sum(test['dataset_id']=='FD00'+str(i)) for i in range(1,5)]\n",
    "    sum(testlengths) == raw_testlen\n",
    "    \n",
    "    ''' MERGING TEST DATA '''\n",
    "    set(test.unit_id) == set(test_left_cycles.unit_id) # checking if the amount of unique engines is the same, so that inner join is meaningful\n",
    "    test_merged = pd.merge(test, test_left_cycles, how='left', on=['dataset_id','unit_id'], sort=False) # OK\n",
    "    \n",
    "    together = [train,test_merged]\n",
    "    fulldata1 = pd.concat(together)\n",
    "    \n",
    "    ''' Relabelling dataset id and reordering columns'''\n",
    "    \n",
    "    fulldata1['dataset_id'] = fulldata1['dataset_id'].str.replace('FD00', '')\n",
    "    fulldata1['dataset_id'] = fulldata1['dataset_id'].apply(num)\n",
    "    fulldata1 = fulldata1.rename(columns={'unit_id':'id'})\n",
    "    fulldata_cols = fulldata1.columns.tolist()\n",
    "    fulldata_cols = fulldata_cols[-1:] + fulldata_cols[:-1]\n",
    "    fulldata1 = fulldata1[fulldata_cols]\n",
    "    ''' Before we do anything else, we need to recode the engine ids since we have 4 datasets with different engines'''\n",
    "    ''' Coding the engines to be unique -- since the engines in the test and train are recoded \n",
    "    in the same fashion, one should be allowed to use engine id as a predictor'''\n",
    "    \n",
    "    def recode_engines(fulldata1): \n",
    "        traininfo = {}\n",
    "        testinfo = {}\n",
    "        train_ = fulldata1[:raw_trainlen]\n",
    "        test_ = fulldata1[raw_trainlen:]\n",
    "    \n",
    "        for i in range(1,5):\n",
    "            indicatorvar = train_.dataset_id == i\n",
    "            train = train_[indicatorvar]\n",
    "        \n",
    "            indicatorvar2 = test_.dataset_id == i\n",
    "            test = test_[indicatorvar2]\n",
    "            if i > 1:\n",
    "                train['id']=train['id'] + max(traininfo[i-1]['id']) # take the maximum engine number\n",
    "                test['id']=test['id'] + max(traininfo[i-1]['id'])\n",
    "            traininfo[i] = train\n",
    "            testinfo[i] = test\n",
    "            \n",
    "            if i == 4:\n",
    "                train_full = pd.concat(traininfo.values())\n",
    "                test_full = pd.concat(testinfo.values())\n",
    "        #if output_len(train_full)==output_len(train):\n",
    "        return([train_full,test_full])\n",
    "    #    else:\n",
    "    #        return('some data missing')\n",
    "            \n",
    "    train_full,test_full = recode_engines(fulldata1)\n",
    "    output_len(train_full)+output_len(test_full) == output_len(fulldata1) # check ok\n",
    "    \n",
    "    sum(train_full.isnull().sum()) == 0 # we see that there are missing values. We know it's rul of train data\n",
    "    sum(test_full.isnull().sum()) == 0  # true\n",
    "    \n",
    "    train_full.isnull().sum() # 160359 missing values standing for train data rul. Next we generate this feature\n",
    "    test_full.isnull().sum()\n",
    "    \n",
    "    ''' In this step we create the remaining useful life variable for the train data'''\n",
    "    rul_frame = pd.DataFrame(train_full.groupby('id')['cycle'].max()).reset_index()\n",
    "    rul_frame.columns = ['id', 'max']\n",
    "    train_full = train_full.merge(rul_frame, on=['id'], how='left')\n",
    "    train_full['rul'] = train_full['max'] - train_full['cycle']\n",
    "    train_full.drop('max', axis=1, inplace=True)\n",
    "    \n",
    "    np.mean((train_full.groupby('id')).min()) # ok average of the min rul's per engine is 0\n",
    "    np.mean((test_full.groupby('id')).min()) # in the test data, the average of min rul's per engine is 81.4\n",
    "    \n",
    "    '''Checking that the train and test data lengths add up to full data'''\n",
    "    output_len(train_full) + output_len(test_full) == raw_full_len # ok\n",
    "    \n",
    "    ''' Now are from hence: ID REFERS TO UNIT ID (ENGINE ID)'''\n",
    "    \n",
    "    full_lengths= [sum(fulldata1['dataset_id']==i) for i in range(1,5)]\n",
    "    sum(full_lengths) == raw_full_len  # data size OK\n",
    "    \n",
    "    ''' check if any missing values in the data, should be 0 now'''\n",
    "    sum(train_full.isnull().sum()) == 0 # we see that there are missing values. We know it's rul of train data\n",
    "    sum(test_full.isnull().sum()) == 0 # ok\n",
    "    ''' \n",
    "    160359 ELEMENTS TRAIN  \n",
    "    104897 ELEMENTS TEST, \n",
    "    265256 ELEMENTS TOTAL '''\n",
    "    ############## TRAINING DATA MOD ################\n",
    "    trainlen = output_len(train_full)\n",
    "    testlen = output_len(test_full)\n",
    "    trainlen\n",
    "    testlen\n",
    "    \n",
    "    ################################################################################3\n",
    "    \n",
    "    frames = [train_full,test_full]\n",
    "    '''combining train and test now should result in 0 NaNs since both have the rul variable'''\n",
    "    fulldata = pd.concat(frames) \n",
    "    \n",
    "    fulldata.isnull().sum() # 'Fine, fulldata has 0 Na\n",
    "    \n",
    "    print(fulldata.head())\n",
    "    print('\\n Data Types:')\n",
    "    print(fulldata.dtypes)\n",
    "    \n",
    "    ' We see that sensors 17 and 18 are of integer type '\n",
    "    \n",
    "    ################################################## DATA NORMALIZATION ###################################3\n",
    "    \n",
    "    '''DATA NORMALIZATION -- we don't use test data when normalizing training data'''\n",
    "    '''train_full.columns returns the index object with feature names for the training set'''\n",
    "    \n",
    "    feats_to_normalize = train_full.columns.difference(['id','cycle','rul','dataset_id']) \n",
    "    ''' exclude engine ID and remaining useful life columns from normalization because engine ID is just an identifier\n",
    "    and remaining useful life should not be normalized to the max of the engines since each engine is different ?'''\n",
    "    \n",
    "    'Only sensor and setting columns are normalized'\n",
    "    \n",
    "    def normalize_data(data,feats_to_normalize):\n",
    "        minmax_scaler = preprocessing.MinMaxScaler()\n",
    "        normalized_data = pd.DataFrame(minmax_scaler.fit_transform(data[feats_to_normalize]), columns=feats_to_normalize, index=data.index)\n",
    "        return(normalized_data)\n",
    "    \n",
    "    def merge_data(full,feats_to_normalize):\n",
    "        normalized_train = normalize_data(full,feats_to_normalize)\n",
    "        normalized_train.isnull().sum() # 'Fine, normalized_train has 0 Na\n",
    "        unnormalized_features = full.columns.difference(feats_to_normalize)\n",
    "        combined = full[unnormalized_features].join(normalized_train)\n",
    "        if 'cycle_norm' in full.columns:\n",
    "            combined.drop('cycle_norm',axis=1)\n",
    "        combined = combined.reindex(columns = full.columns)\n",
    "        return(combined)\n",
    "        \n",
    "    processed_train = merge_data(train_full,feats_to_normalize)\n",
    "    processed_test = merge_data(test_full,feats_to_normalize)\n",
    "    output_len(processed_train) + output_len(processed_test) == output_len(fulldata1) # check ok\n",
    "    return(processed_train,processed_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\BCI-EXPERT\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:88: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "C:\\Users\\BCI-EXPERT\\Anaconda3\\lib\\site-packages\\ipykernel\\__main__.py:89: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   id  cycle  dataset_id    rul  sensor 1  sensor 10  sensor 11  sensor 12  \\\n",
      "0   1      1           1  191.0    518.67        1.3      47.47     521.66   \n",
      "1   1      2           1  190.0    518.67        1.3      47.49     522.28   \n",
      "2   1      3           1  189.0    518.67        1.3      47.27     522.42   \n",
      "3   1      4           1  188.0    518.67        1.3      47.13     522.86   \n",
      "4   1      5           1  187.0    518.67        1.3      47.28     522.19   \n",
      "\n",
      "   sensor 13  sensor 14    ...      sensor 3  sensor 4  sensor 5  sensor 6  \\\n",
      "0    2388.02    8138.62    ...       1589.70   1400.60     14.62     21.61   \n",
      "1    2388.07    8131.49    ...       1591.82   1403.14     14.62     21.61   \n",
      "2    2388.03    8133.23    ...       1587.99   1404.20     14.62     21.61   \n",
      "3    2388.08    8133.83    ...       1582.79   1401.87     14.62     21.61   \n",
      "4    2388.04    8133.80    ...       1582.85   1406.22     14.62     21.61   \n",
      "\n",
      "   sensor 7  sensor 8  sensor 9  setting 1  setting 2  setting 3  \n",
      "0    554.36   2388.06   9046.19    -0.0007    -0.0004      100.0  \n",
      "1    553.75   2388.04   9044.07     0.0019    -0.0003      100.0  \n",
      "2    554.26   2388.08   9052.94    -0.0043     0.0003      100.0  \n",
      "3    554.45   2388.11   9049.48     0.0007     0.0000      100.0  \n",
      "4    554.00   2388.06   9055.15    -0.0019    -0.0002      100.0  \n",
      "\n",
      "[5 rows x 28 columns]\n",
      "\n",
      " Data Types:\n",
      "id              int64\n",
      "cycle           int64\n",
      "dataset_id      int64\n",
      "rul           float64\n",
      "sensor 1      float64\n",
      "sensor 10     float64\n",
      "sensor 11     float64\n",
      "sensor 12     float64\n",
      "sensor 13     float64\n",
      "sensor 14     float64\n",
      "sensor 15     float64\n",
      "sensor 16     float64\n",
      "sensor 17       int64\n",
      "sensor 18       int64\n",
      "sensor 19     float64\n",
      "sensor 2      float64\n",
      "sensor 20     float64\n",
      "sensor 21     float64\n",
      "sensor 3      float64\n",
      "sensor 4      float64\n",
      "sensor 5      float64\n",
      "sensor 6      float64\n",
      "sensor 7      float64\n",
      "sensor 8      float64\n",
      "sensor 9      float64\n",
      "setting 1     float64\n",
      "setting 2     float64\n",
      "setting 3     float64\n",
      "dtype: object\n"
     ]
    }
   ],
   "source": [
    "processed_train,processed_test = do_data_processing(datadir2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>cycle</th>\n",
       "      <th>dataset_id</th>\n",
       "      <th>rul</th>\n",
       "      <th>sensor 1</th>\n",
       "      <th>sensor 10</th>\n",
       "      <th>sensor 11</th>\n",
       "      <th>sensor 12</th>\n",
       "      <th>sensor 13</th>\n",
       "      <th>sensor 14</th>\n",
       "      <th>...</th>\n",
       "      <th>sensor 3</th>\n",
       "      <th>sensor 4</th>\n",
       "      <th>sensor 5</th>\n",
       "      <th>sensor 6</th>\n",
       "      <th>sensor 7</th>\n",
       "      <th>sensor 8</th>\n",
       "      <th>sensor 9</th>\n",
       "      <th>setting 1</th>\n",
       "      <th>setting 2</th>\n",
       "      <th>setting 3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>191</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.948718</td>\n",
       "      <td>0.915132</td>\n",
       "      <td>0.961313</td>\n",
       "      <td>0.993194</td>\n",
       "      <td>0.653748</td>\n",
       "      <td>...</td>\n",
       "      <td>0.927293</td>\n",
       "      <td>0.902111</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.962153</td>\n",
       "      <td>0.998776</td>\n",
       "      <td>0.842550</td>\n",
       "      <td>0.000190</td>\n",
       "      <td>0.000237</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>190</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.948718</td>\n",
       "      <td>0.916733</td>\n",
       "      <td>0.962828</td>\n",
       "      <td>0.993332</td>\n",
       "      <td>0.637831</td>\n",
       "      <td>...</td>\n",
       "      <td>0.932957</td>\n",
       "      <td>0.908192</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.960749</td>\n",
       "      <td>0.998734</td>\n",
       "      <td>0.840867</td>\n",
       "      <td>0.000252</td>\n",
       "      <td>0.000356</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>189</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.948718</td>\n",
       "      <td>0.899119</td>\n",
       "      <td>0.963170</td>\n",
       "      <td>0.993222</td>\n",
       "      <td>0.641715</td>\n",
       "      <td>...</td>\n",
       "      <td>0.922723</td>\n",
       "      <td>0.910730</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.961923</td>\n",
       "      <td>0.998818</td>\n",
       "      <td>0.847906</td>\n",
       "      <td>0.000105</td>\n",
       "      <td>0.001068</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>188</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.948718</td>\n",
       "      <td>0.887910</td>\n",
       "      <td>0.964246</td>\n",
       "      <td>0.993359</td>\n",
       "      <td>0.643055</td>\n",
       "      <td>...</td>\n",
       "      <td>0.908829</td>\n",
       "      <td>0.905152</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.962360</td>\n",
       "      <td>0.998882</td>\n",
       "      <td>0.845161</td>\n",
       "      <td>0.000224</td>\n",
       "      <td>0.000712</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>187</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.948718</td>\n",
       "      <td>0.899920</td>\n",
       "      <td>0.962608</td>\n",
       "      <td>0.993249</td>\n",
       "      <td>0.642988</td>\n",
       "      <td>...</td>\n",
       "      <td>0.908989</td>\n",
       "      <td>0.915565</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.961324</td>\n",
       "      <td>0.998776</td>\n",
       "      <td>0.849660</td>\n",
       "      <td>0.000162</td>\n",
       "      <td>0.000475</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  cycle  dataset_id  rul  sensor 1  sensor 10  sensor 11  sensor 12  \\\n",
       "0   1      1           1  191       1.0   0.948718   0.915132   0.961313   \n",
       "1   1      2           1  190       1.0   0.948718   0.916733   0.962828   \n",
       "2   1      3           1  189       1.0   0.948718   0.899119   0.963170   \n",
       "3   1      4           1  188       1.0   0.948718   0.887910   0.964246   \n",
       "4   1      5           1  187       1.0   0.948718   0.899920   0.962608   \n",
       "\n",
       "   sensor 13  sensor 14    ...      sensor 3  sensor 4  sensor 5  sensor 6  \\\n",
       "0   0.993194   0.653748    ...      0.927293  0.902111       1.0       1.0   \n",
       "1   0.993332   0.637831    ...      0.932957  0.908192       1.0       1.0   \n",
       "2   0.993222   0.641715    ...      0.922723  0.910730       1.0       1.0   \n",
       "3   0.993359   0.643055    ...      0.908829  0.905152       1.0       1.0   \n",
       "4   0.993249   0.642988    ...      0.908989  0.915565       1.0       1.0   \n",
       "\n",
       "   sensor 7  sensor 8  sensor 9  setting 1  setting 2  setting 3  \n",
       "0  0.962153  0.998776  0.842550   0.000190   0.000237        1.0  \n",
       "1  0.960749  0.998734  0.840867   0.000252   0.000356        1.0  \n",
       "2  0.961923  0.998818  0.847906   0.000105   0.001068        1.0  \n",
       "3  0.962360  0.998882  0.845161   0.000224   0.000712        1.0  \n",
       "4  0.961324  0.998776  0.849660   0.000162   0.000475        1.0  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processed_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>cycle</th>\n",
       "      <th>dataset_id</th>\n",
       "      <th>rul</th>\n",
       "      <th>sensor 1</th>\n",
       "      <th>sensor 10</th>\n",
       "      <th>sensor 11</th>\n",
       "      <th>sensor 12</th>\n",
       "      <th>sensor 13</th>\n",
       "      <th>sensor 14</th>\n",
       "      <th>...</th>\n",
       "      <th>sensor 3</th>\n",
       "      <th>sensor 4</th>\n",
       "      <th>sensor 5</th>\n",
       "      <th>sensor 6</th>\n",
       "      <th>sensor 7</th>\n",
       "      <th>sensor 8</th>\n",
       "      <th>sensor 9</th>\n",
       "      <th>setting 1</th>\n",
       "      <th>setting 2</th>\n",
       "      <th>setting 3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>57810</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>112.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.948718</td>\n",
       "      <td>0.911837</td>\n",
       "      <td>0.963560</td>\n",
       "      <td>0.994672</td>\n",
       "      <td>0.701119</td>\n",
       "      <td>...</td>\n",
       "      <td>0.933831</td>\n",
       "      <td>0.912248</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.964743</td>\n",
       "      <td>0.998923</td>\n",
       "      <td>0.891455</td>\n",
       "      <td>0.000262</td>\n",
       "      <td>0.001068</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57811</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>112.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.948718</td>\n",
       "      <td>0.936327</td>\n",
       "      <td>0.964637</td>\n",
       "      <td>0.994755</td>\n",
       "      <td>0.736893</td>\n",
       "      <td>...</td>\n",
       "      <td>0.942400</td>\n",
       "      <td>0.905430</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.966937</td>\n",
       "      <td>0.998860</td>\n",
       "      <td>0.895018</td>\n",
       "      <td>0.000143</td>\n",
       "      <td>0.000356</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57812</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>112.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.948718</td>\n",
       "      <td>0.936327</td>\n",
       "      <td>0.964172</td>\n",
       "      <td>0.994672</td>\n",
       "      <td>0.712688</td>\n",
       "      <td>...</td>\n",
       "      <td>0.938305</td>\n",
       "      <td>0.919896</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.965228</td>\n",
       "      <td>0.998944</td>\n",
       "      <td>0.897148</td>\n",
       "      <td>0.000214</td>\n",
       "      <td>0.000831</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57813</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>112.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.948718</td>\n",
       "      <td>0.918367</td>\n",
       "      <td>0.962727</td>\n",
       "      <td>0.994727</td>\n",
       "      <td>0.719807</td>\n",
       "      <td>...</td>\n",
       "      <td>0.930658</td>\n",
       "      <td>0.932310</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.965136</td>\n",
       "      <td>0.998902</td>\n",
       "      <td>0.887363</td>\n",
       "      <td>0.000307</td>\n",
       "      <td>0.000712</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57814</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>112.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.948718</td>\n",
       "      <td>0.920816</td>\n",
       "      <td>0.964613</td>\n",
       "      <td>0.994672</td>\n",
       "      <td>0.711264</td>\n",
       "      <td>...</td>\n",
       "      <td>0.938983</td>\n",
       "      <td>0.921314</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.965344</td>\n",
       "      <td>0.998860</td>\n",
       "      <td>0.886742</td>\n",
       "      <td>0.000240</td>\n",
       "      <td>0.000712</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  cycle  dataset_id    rul  sensor 1  sensor 10  sensor 11  \\\n",
       "57810   1      1           1  112.0       1.0   0.948718   0.911837   \n",
       "57811   1      2           1  112.0       1.0   0.948718   0.936327   \n",
       "57812   1      3           1  112.0       1.0   0.948718   0.936327   \n",
       "57813   1      4           1  112.0       1.0   0.948718   0.918367   \n",
       "57814   1      5           1  112.0       1.0   0.948718   0.920816   \n",
       "\n",
       "       sensor 12  sensor 13  sensor 14    ...      sensor 3  sensor 4  \\\n",
       "57810   0.963560   0.994672   0.701119    ...      0.933831  0.912248   \n",
       "57811   0.964637   0.994755   0.736893    ...      0.942400  0.905430   \n",
       "57812   0.964172   0.994672   0.712688    ...      0.938305  0.919896   \n",
       "57813   0.962727   0.994727   0.719807    ...      0.930658  0.932310   \n",
       "57814   0.964613   0.994672   0.711264    ...      0.938983  0.921314   \n",
       "\n",
       "       sensor 5  sensor 6  sensor 7  sensor 8  sensor 9  setting 1  setting 2  \\\n",
       "57810       1.0       1.0  0.964743  0.998923  0.891455   0.000262   0.001068   \n",
       "57811       1.0       1.0  0.966937  0.998860  0.895018   0.000143   0.000356   \n",
       "57812       1.0       1.0  0.965228  0.998944  0.897148   0.000214   0.000831   \n",
       "57813       1.0       1.0  0.965136  0.998902  0.887363   0.000307   0.000712   \n",
       "57814       1.0       1.0  0.965344  0.998860  0.886742   0.000240   0.000712   \n",
       "\n",
       "       setting 3  \n",
       "57810        1.0  \n",
       "57811        1.0  \n",
       "57812        1.0  \n",
       "57813        1.0  \n",
       "57814        1.0  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processed_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot Test Data Correlation Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "size = 25\n",
    "rawtest = pd.read_csv('./data/test.csv',engine='python')\n",
    "rawtest_timeseries =  rawtest.iloc[:, [0,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28]] # index buggy?\n",
    "grouped = rawtest_timeseries.groupby('id')\n",
    "\n",
    "corrmatrix = rawtest_timeseries.corr()\n",
    "fig, ax = plt.subplots(figsize=(size, size))\n",
    "ax.matshow(corrmatrix)\n",
    "plt.xticks(range(len(corrmatrix.columns)), corrmatrix.columns);\n",
    "plt.yticks(range(len(corrmatrix.columns)), corrmatrix.columns);\n",
    "fig.savefig(\"./timeseries_correlationmatrix.png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Time Series Features Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn import cross_validation as cv\n",
    "\n",
    "min_max_cycle_test= np.min(processed_test.groupby('id')['cycle'].max()) \n",
    "\n",
    "min_max_cycle_test= np.min(processed_test.groupby('id')['cycle'].max()) # NB! Smallest window in the testing set is 19, so cannot use longer windows than 18\n",
    "\n",
    "\n",
    "test_feature_names  = list(filter(lambda s: (not s.startswith('dataset') and not s.startswith('lessthan') and not s.startswith('rul')), processed_test.columns))\n",
    "\n",
    "def prepare_features(data,windowlength,features, iftest):\n",
    "    def generate_features(data, windowlength, features):\n",
    "        data_array = data[features].values\n",
    "        data_len = data_array.shape[0]\n",
    "        for start, end in zip(range(0, data_len-windowlength), range(windowlength, data_len)):\n",
    "            yield data_array[start:end, :]\n",
    "        return data_array\n",
    "        \n",
    "    def generate_labels(data, windowlength, label):\n",
    "        data_array = data[label].values\n",
    "        data_length = data_array.shape[0]\n",
    "        return data_array[windowlength:data_length, :]\n",
    "    \n",
    "    listout = list()\n",
    "    if iftest==1:\n",
    "        \n",
    "        initial_labels = data['rul']\n",
    "        test_array = data[features]\n",
    "        \n",
    "        cv_array, red_test_array, cv_labels, red_test_labels = cv.train_test_split(test_array,initial_labels, test_size=0.5, random_state=1)\n",
    "        \n",
    "        cv_generator= (list(generate_features(data[data['id']==id], windowlength, features)) for id in data['id'].unique())\n",
    "\n",
    "        test_generator= (list(generate_features(data[data['id']==id], windowlength, features)) for id in data['id'].unique())\n",
    "        \n",
    "        test_features = np.concatenate(list(test_generator)).astype(np.float32)\n",
    "        cv_features = np.concatenate(list(cv_generator)).astype(np.float32)\n",
    "\n",
    "        \n",
    "        cv_labels = [generate_labels(data[data['id']==id], windowlength, ['rul']) for id in data['id'].unique()]\n",
    "        cv_labels = np.concatenate(cv_labels).astype(np.float32)\n",
    "        \n",
    "        test_labels = [generate_labels(data[data['id']==id], windowlength, ['rul']) for id in data['id'].unique()]\n",
    "        test_labels = np.concatenate(test_labels).astype(np.float32)\n",
    "        listout = [cv_features,cv_labels,test_features,test_labels]\n",
    "        \n",
    "    else:\n",
    "        train_generator =  (list(generate_features(data[data['id']==id], windowlength, \\\n",
    "                                                   features)) for id in data['id'].unique())\n",
    "        train_features = np.concatenate(list(train_generator)).astype(np.float32)  \n",
    "        train_labels = [generate_labels(data[data['id']==id], windowlength, ['rul']) for id in data['id'].unique()]\n",
    "        train_labels = np.concatenate(train_labels).astype(np.float32)\n",
    "        listout = [train_features,train_labels]\n",
    "  \n",
    "    return listout\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "OSError",
     "evalue": "[Errno 28] No space left on device",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-33-b2463d974f2c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      9\u001b[0m     \u001b[0mcv_features\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcv_labels\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtest_features\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtest_labels\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mouttest\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mouttest\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mouttest\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mouttest\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'wb'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m         \u001b[0mpickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtrain_features\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtrain_labels\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtest_features\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtest_labels\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcv_features\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcv_labels\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mOSError\u001b[0m: [Errno 28] No space left on device"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "test_feature_names  = list(filter(lambda s: (not s.startswith('dataset') and not s.startswith('lessthan') and not s.startswith('rul')), processed_test.columns))\n",
    "\n",
    "for windowlength in [5,18]: \n",
    "    filename = 'datasets_final' + 'windowlen' + str(windowlength)\n",
    "    outtrain = prepare_features(processed_train,windowlength,test_feature_names, 0)\n",
    "    outtest = prepare_features(processed_test,windowlength,test_feature_names, 1)\n",
    "    train_features,train_labels = outtrain[0],outtrain[1]\n",
    "    cv_features,cv_labels,test_features,test_labels = outtest[0],outtest[1],outtest[2],outtest[3]\n",
    "    with open(filename, 'wb') as f:\n",
    "        pickle.dump([train_features,train_labels,test_features,test_labels,cv_features,cv_labels], f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Machine Learning Modelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "lstm_3 (LSTM)                (None, 5, 32)             7552      \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 5, 32)             0         \n",
      "_________________________________________________________________\n",
      "lstm_4 (LSTM)                (None, 20)                4240      \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 20)                0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 21        \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 11,813\n",
      "Trainable params: 11,813\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 148973 samples, validate on 7841 samples\n",
      "Epoch 1/30\n",
      "Epoch 00001: val_loss improved from inf to 17460.30111, saving model to ./model_window5hidden32_20dropout0.22batch200.h5\n",
      " - 14s - loss: 18243.6865 - mean_squared_error: 18243.6865 - mean_absolute_error: 107.7474 - val_loss: 17460.3011 - val_mean_squared_error: 17460.3011 - val_mean_absolute_error: 106.0406\n",
      "Epoch 2/30\n",
      "Epoch 00002: val_loss improved from 17460.30111 to 14465.27907, saving model to ./model_window5hidden32_20dropout0.22batch200.h5\n",
      " - 11s - loss: 15119.9834 - mean_squared_error: 15119.9834 - mean_absolute_error: 95.0121 - val_loss: 14465.2791 - val_mean_squared_error: 14465.2791 - val_mean_absolute_error: 94.3814\n",
      "Epoch 3/30\n",
      "Epoch 00003: val_loss improved from 14465.27907 to 11958.90658, saving model to ./model_window5hidden32_20dropout0.22batch200.h5\n",
      " - 13s - loss: 12550.3028 - mean_squared_error: 12550.3028 - mean_absolute_error: 84.8045 - val_loss: 11958.9066 - val_mean_squared_error: 11958.9066 - val_mean_absolute_error: 84.7753\n",
      "Epoch 4/30\n",
      "Epoch 00004: val_loss improved from 11958.90658 to 9938.66998, saving model to ./model_window5hidden32_20dropout0.22batch200.h5\n",
      " - 12s - loss: 10469.5637 - mean_squared_error: 10469.5637 - mean_absolute_error: 76.7881 - val_loss: 9938.6700 - val_mean_squared_error: 9938.6700 - val_mean_absolute_error: 77.2142\n",
      "Epoch 5/30\n",
      "Epoch 00005: val_loss improved from 9938.66998 to 8406.66730, saving model to ./model_window5hidden32_20dropout0.22batch200.h5\n",
      " - 12s - loss: 8883.7065 - mean_squared_error: 8883.7065 - mean_absolute_error: 70.9533 - val_loss: 8406.6673 - val_mean_squared_error: 8406.6673 - val_mean_absolute_error: 71.7004\n",
      "Epoch 6/30\n",
      "Epoch 00006: val_loss improved from 8406.66730 to 7365.02157, saving model to ./model_window5hidden32_20dropout0.22batch200.h5\n",
      " - 11s - loss: 7783.6757 - mean_squared_error: 7783.6757 - mean_absolute_error: 67.3664 - val_loss: 7365.0216 - val_mean_squared_error: 7365.0216 - val_mean_absolute_error: 68.2328\n",
      "Epoch 7/30\n",
      "Epoch 00007: val_loss improved from 7365.02157 to 6820.86082, saving model to ./model_window5hidden32_20dropout0.22batch200.h5\n",
      " - 11s - loss: 7183.9095 - mean_squared_error: 7183.9095 - mean_absolute_error: 65.9879 - val_loss: 6820.8608 - val_mean_squared_error: 6820.8608 - val_mean_absolute_error: 66.7580\n",
      "Epoch 8/30\n",
      "Epoch 00008: val_loss improved from 6820.86082 to 6693.30261, saving model to ./model_window5hidden32_20dropout0.22batch200.h5\n",
      " - 12s - loss: 7002.0525 - mean_squared_error: 7002.0525 - mean_absolute_error: 66.0440 - val_loss: 6693.3026 - val_mean_squared_error: 6693.3026 - val_mean_absolute_error: 66.5881\n",
      "Epoch 9/30\n",
      "Epoch 00009: val_loss improved from 6693.30261 to 6683.24704, saving model to ./model_window5hidden32_20dropout0.22batch200.h5\n",
      " - 11s - loss: 6989.6743 - mean_squared_error: 6989.6743 - mean_absolute_error: 66.2718 - val_loss: 6683.2470 - val_mean_squared_error: 6683.2470 - val_mean_absolute_error: 66.5879\n",
      "Epoch 10/30\n",
      "Epoch 00010: val_loss improved from 6683.24704 to 5294.87814, saving model to ./model_window5hidden32_20dropout0.22batch200.h5\n",
      " - 13s - loss: 6422.0455 - mean_squared_error: 6422.0455 - mean_absolute_error: 61.6181 - val_loss: 5294.8781 - val_mean_squared_error: 5294.8781 - val_mean_absolute_error: 56.6062\n",
      "Epoch 11/30\n",
      "Epoch 00011: val_loss improved from 5294.87814 to 4587.70812, saving model to ./model_window5hidden32_20dropout0.22batch200.h5\n",
      " - 12s - loss: 5179.2983 - mean_squared_error: 5179.2983 - mean_absolute_error: 52.3440 - val_loss: 4587.7081 - val_mean_squared_error: 4587.7081 - val_mean_absolute_error: 52.7850\n",
      "Epoch 12/30\n",
      "Epoch 00012: val_loss improved from 4587.70812 to 4127.23232, saving model to ./model_window5hidden32_20dropout0.22batch200.h5\n",
      " - 12s - loss: 4758.4983 - mean_squared_error: 4758.4983 - mean_absolute_error: 50.3775 - val_loss: 4127.2323 - val_mean_squared_error: 4127.2323 - val_mean_absolute_error: 50.6670\n",
      "Epoch 13/30\n",
      "Epoch 00013: val_loss improved from 4127.23232 to 3797.20454, saving model to ./model_window5hidden32_20dropout0.22batch200.h5\n",
      " - 11s - loss: 4456.6145 - mean_squared_error: 4456.6145 - mean_absolute_error: 49.1481 - val_loss: 3797.2045 - val_mean_squared_error: 3797.2045 - val_mean_absolute_error: 49.0518\n",
      "Epoch 14/30\n",
      "Epoch 00014: val_loss did not improve\n",
      " - 11s - loss: 4296.2783 - mean_squared_error: 4296.2783 - mean_absolute_error: 48.6037 - val_loss: 3875.3147 - val_mean_squared_error: 3875.3147 - val_mean_absolute_error: 50.5620\n",
      "Epoch 15/30\n",
      "Epoch 00015: val_loss improved from 3797.20454 to 3558.56029, saving model to ./model_window5hidden32_20dropout0.22batch200.h5\n",
      " - 11s - loss: 4191.9394 - mean_squared_error: 4191.9394 - mean_absolute_error: 48.2457 - val_loss: 3558.5603 - val_mean_squared_error: 3558.5603 - val_mean_absolute_error: 48.3073\n",
      "Epoch 16/30\n",
      "Epoch 00016: val_loss improved from 3558.56029 to 3429.12304, saving model to ./model_window5hidden32_20dropout0.22batch200.h5\n",
      " - 12s - loss: 4148.1457 - mean_squared_error: 4148.1457 - mean_absolute_error: 48.1544 - val_loss: 3429.1230 - val_mean_squared_error: 3429.1230 - val_mean_absolute_error: 47.4970\n",
      "Epoch 17/30\n",
      "Epoch 00017: val_loss did not improve\n",
      " - 13s - loss: 4120.1283 - mean_squared_error: 4120.1283 - mean_absolute_error: 48.0475 - val_loss: 3537.5297 - val_mean_squared_error: 3537.5297 - val_mean_absolute_error: 48.5435\n",
      "Epoch 18/30\n",
      "Epoch 00018: val_loss improved from 3429.12304 to 3271.77879, saving model to ./model_window5hidden32_20dropout0.22batch200.h5\n",
      " - 12s - loss: 4089.2274 - mean_squared_error: 4089.2274 - mean_absolute_error: 47.9410 - val_loss: 3271.7788 - val_mean_squared_error: 3271.7788 - val_mean_absolute_error: 46.4184\n",
      "Epoch 19/30\n",
      "Epoch 00019: val_loss did not improve\n",
      " - 13s - loss: 4074.4856 - mean_squared_error: 4074.4856 - mean_absolute_error: 47.8921 - val_loss: 3868.4010 - val_mean_squared_error: 3868.4010 - val_mean_absolute_error: 51.8973\n",
      "Epoch 20/30\n",
      "Epoch 00020: val_loss did not improve\n",
      " - 12s - loss: 4055.7218 - mean_squared_error: 4055.7218 - mean_absolute_error: 47.7559 - val_loss: 3555.4776 - val_mean_squared_error: 3555.4776 - val_mean_absolute_error: 49.2776\n",
      "Epoch 21/30\n",
      "Epoch 00021: val_loss did not improve\n",
      " - 11s - loss: 4047.2257 - mean_squared_error: 4047.2257 - mean_absolute_error: 47.7298 - val_loss: 3511.2976 - val_mean_squared_error: 3511.2976 - val_mean_absolute_error: 49.1179\n",
      "Epoch 22/30\n",
      "Epoch 00022: val_loss did not improve\n",
      " - 11s - loss: 4030.6592 - mean_squared_error: 4030.6592 - mean_absolute_error: 47.5788 - val_loss: 3686.7367 - val_mean_squared_error: 3686.7367 - val_mean_absolute_error: 50.8124\n",
      "Epoch 23/30\n",
      "Epoch 00023: val_loss did not improve\n",
      " - 12s - loss: 4008.1302 - mean_squared_error: 4008.1302 - mean_absolute_error: 47.4770 - val_loss: 3568.1498 - val_mean_squared_error: 3568.1498 - val_mean_absolute_error: 49.8004\n",
      "Epoch 24/30\n",
      "Epoch 00024: val_loss did not improve\n",
      " - 11s - loss: 3999.7921 - mean_squared_error: 3999.7921 - mean_absolute_error: 47.3801 - val_loss: 3479.2709 - val_mean_squared_error: 3479.2709 - val_mean_absolute_error: 43.4856\n",
      "Epoch 25/30\n",
      "Epoch 00025: val_loss did not improve\n",
      " - 11s - loss: 3985.6833 - mean_squared_error: 3985.6833 - mean_absolute_error: 47.3081 - val_loss: 3667.8419 - val_mean_squared_error: 3667.8419 - val_mean_absolute_error: 50.7261\n",
      "Epoch 26/30\n",
      "Epoch 00026: val_loss did not improve\n",
      " - 11s - loss: 3975.1225 - mean_squared_error: 3975.1225 - mean_absolute_error: 47.2515 - val_loss: 3338.6143 - val_mean_squared_error: 3338.6143 - val_mean_absolute_error: 47.9253\n",
      "Epoch 27/30\n",
      "Epoch 00027: val_loss did not improve\n",
      " - 11s - loss: 3968.3731 - mean_squared_error: 3968.3731 - mean_absolute_error: 47.1488 - val_loss: 3775.1213 - val_mean_squared_error: 3775.1213 - val_mean_absolute_error: 51.6286\n",
      "Epoch 28/30\n",
      "Epoch 00028: val_loss did not improve\n",
      " - 12s - loss: 3962.3992 - mean_squared_error: 3962.3992 - mean_absolute_error: 47.1290 - val_loss: 3518.6160 - val_mean_squared_error: 3518.6160 - val_mean_absolute_error: 49.6193\n",
      "Epoch 00028: early stopping\n",
      "dict_keys(['val_mean_squared_error', 'mean_absolute_error', 'loss', 'val_loss', 'val_mean_absolute_error', 'mean_squared_error'])\n",
      "101362/101362 [==============================] - 3s 27us/step\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "lstm_5 (LSTM)                (None, 5, 16)             2752      \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 5, 16)             0         \n",
      "_________________________________________________________________\n",
      "lstm_6 (LSTM)                (None, 20)                2960      \n",
      "_________________________________________________________________\n",
      "dropout_6 (Dropout)          (None, 20)                0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 1)                 21        \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 5,733\n",
      "Trainable params: 5,733\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 148973 samples, validate on 7841 samples\n",
      "Epoch 1/30\n",
      "Epoch 00001: val_loss improved from inf to 17310.67991, saving model to ./model_window5hidden16_20dropout0.22batch200.h5\n",
      " - 13s - loss: 18119.0642 - mean_squared_error: 18119.0642 - mean_absolute_error: 107.2514 - val_loss: 17310.6799 - val_mean_squared_error: 17310.6799 - val_mean_absolute_error: 105.4554\n",
      "Epoch 2/30\n",
      "Epoch 00002: val_loss improved from 17310.67991 to 14338.84317, saving model to ./model_window5hidden16_20dropout0.22batch200.h5\n",
      " - 12s - loss: 14997.6173 - mean_squared_error: 14997.6173 - mean_absolute_error: 94.5127 - val_loss: 14338.8432 - val_mean_squared_error: 14338.8432 - val_mean_absolute_error: 93.8927\n",
      "Epoch 3/30\n",
      "Epoch 00003: val_loss improved from 14338.84317 to 11854.56645, saving model to ./model_window5hidden16_20dropout0.22batch200.h5\n",
      " - 11s - loss: 12449.1164 - mean_squared_error: 12449.1164 - mean_absolute_error: 84.3861 - val_loss: 11854.5665 - val_mean_squared_error: 11854.5665 - val_mean_absolute_error: 84.3799\n",
      "Epoch 4/30\n",
      "Epoch 00004: val_loss improved from 11854.56645 to 9857.30704, saving model to ./model_window5hidden16_20dropout0.22batch200.h5\n",
      " - 10s - loss: 10381.9807 - mean_squared_error: 10381.9807 - mean_absolute_error: 76.4338 - val_loss: 9857.3070 - val_mean_squared_error: 9857.3070 - val_mean_absolute_error: 76.9150\n",
      "Epoch 5/30\n",
      "Epoch 00005: val_loss improved from 9857.30704 to 8349.01979, saving model to ./model_window5hidden16_20dropout0.22batch200.h5\n",
      " - 10s - loss: 8814.5700 - mean_squared_error: 8814.5700 - mean_absolute_error: 70.7400 - val_loss: 8349.0198 - val_mean_squared_error: 8349.0198 - val_mean_absolute_error: 71.5006\n",
      "Epoch 6/30\n",
      "Epoch 00006: val_loss improved from 8349.01979 to 7331.12764, saving model to ./model_window5hidden16_20dropout0.22batch200.h5\n",
      " - 10s - loss: 7750.9055 - mean_squared_error: 7750.9055 - mean_absolute_error: 67.2823 - val_loss: 7331.1276 - val_mean_squared_error: 7331.1276 - val_mean_absolute_error: 68.1293\n",
      "Epoch 7/30\n",
      "Epoch 00007: val_loss improved from 7331.12764 to 6807.14445, saving model to ./model_window5hidden16_20dropout0.22batch200.h5\n",
      " - 10s - loss: 7178.3005 - mean_squared_error: 7178.3005 - mean_absolute_error: 66.0085 - val_loss: 6807.1444 - val_mean_squared_error: 6807.1444 - val_mean_absolute_error: 66.7305\n",
      "Epoch 8/30\n",
      "Epoch 00008: val_loss improved from 6807.14445 to 6695.84985, saving model to ./model_window5hidden16_20dropout0.22batch200.h5\n",
      " - 10s - loss: 7004.4509 - mean_squared_error: 7004.4509 - mean_absolute_error: 66.1011 - val_loss: 6695.8498 - val_mean_squared_error: 6695.8498 - val_mean_absolute_error: 66.5885\n",
      "Epoch 9/30\n",
      "Epoch 00009: val_loss improved from 6695.84985 to 6683.28640, saving model to ./model_window5hidden16_20dropout0.22batch200.h5\n",
      " - 11s - loss: 6987.3465 - mean_squared_error: 6987.3465 - mean_absolute_error: 66.2784 - val_loss: 6683.2864 - val_mean_squared_error: 6683.2864 - val_mean_absolute_error: 66.5879\n",
      "Epoch 10/30\n",
      "Epoch 00010: val_loss improved from 6683.28640 to 6063.48299, saving model to ./model_window5hidden16_20dropout0.22batch200.h5\n",
      " - 11s - loss: 6876.7330 - mean_squared_error: 6876.7330 - mean_absolute_error: 65.3998 - val_loss: 6063.4830 - val_mean_squared_error: 6063.4830 - val_mean_absolute_error: 62.6513\n",
      "Epoch 11/30\n",
      "Epoch 00011: val_loss improved from 6063.48299 to 5234.55443, saving model to ./model_window5hidden16_20dropout0.22batch200.h5\n",
      " - 10s - loss: 5455.3883 - mean_squared_error: 5455.3883 - mean_absolute_error: 54.2914 - val_loss: 5234.5544 - val_mean_squared_error: 5234.5544 - val_mean_absolute_error: 58.0197\n",
      "Epoch 12/30\n",
      "Epoch 00012: val_loss improved from 5234.55443 to 4339.86166, saving model to ./model_window5hidden16_20dropout0.22batch200.h5\n",
      " - 10s - loss: 4883.5929 - mean_squared_error: 4883.5929 - mean_absolute_error: 51.1167 - val_loss: 4339.8617 - val_mean_squared_error: 4339.8617 - val_mean_absolute_error: 52.2343\n",
      "Epoch 13/30\n",
      "Epoch 00013: val_loss improved from 4339.86166 to 4021.74160, saving model to ./model_window5hidden16_20dropout0.22batch200.h5\n",
      " - 9s - loss: 4545.3259 - mean_squared_error: 4545.3259 - mean_absolute_error: 49.5668 - val_loss: 4021.7416 - val_mean_squared_error: 4021.7416 - val_mean_absolute_error: 50.7730\n",
      "Epoch 14/30\n",
      "Epoch 00014: val_loss improved from 4021.74160 to 3858.41947, saving model to ./model_window5hidden16_20dropout0.22batch200.h5\n",
      " - 10s - loss: 4340.8387 - mean_squared_error: 4340.8387 - mean_absolute_error: 48.7488 - val_loss: 3858.4195 - val_mean_squared_error: 3858.4195 - val_mean_absolute_error: 50.1825\n",
      "Epoch 15/30\n",
      "Epoch 00015: val_loss improved from 3858.41947 to 3689.35609, saving model to ./model_window5hidden16_20dropout0.22batch200.h5\n",
      " - 10s - loss: 4233.9159 - mean_squared_error: 4233.9159 - mean_absolute_error: 48.4288 - val_loss: 3689.3561 - val_mean_squared_error: 3689.3561 - val_mean_absolute_error: 49.0247\n",
      "Epoch 16/30\n",
      "Epoch 00016: val_loss improved from 3689.35609 to 3532.76918, saving model to ./model_window5hidden16_20dropout0.22batch200.h5\n",
      " - 10s - loss: 4170.1057 - mean_squared_error: 4170.1057 - mean_absolute_error: 48.2817 - val_loss: 3532.7692 - val_mean_squared_error: 3532.7692 - val_mean_absolute_error: 48.3232\n",
      "Epoch 17/30\n",
      "Epoch 00017: val_loss did not improve\n",
      " - 11s - loss: 4142.7160 - mean_squared_error: 4142.7160 - mean_absolute_error: 48.2362 - val_loss: 3717.0961 - val_mean_squared_error: 3717.0961 - val_mean_absolute_error: 50.1549\n",
      "Epoch 18/30\n",
      "Epoch 00018: val_loss improved from 3532.76918 to 3302.99318, saving model to ./model_window5hidden16_20dropout0.22batch200.h5\n",
      " - 10s - loss: 4114.4218 - mean_squared_error: 4114.4218 - mean_absolute_error: 48.1607 - val_loss: 3302.9932 - val_mean_squared_error: 3302.9932 - val_mean_absolute_error: 46.2680\n",
      "Epoch 19/30\n",
      "Epoch 00019: val_loss did not improve\n",
      " - 9s - loss: 4103.5430 - mean_squared_error: 4103.5430 - mean_absolute_error: 48.0979 - val_loss: 3504.7337 - val_mean_squared_error: 3504.7337 - val_mean_absolute_error: 48.5913\n",
      "Epoch 20/30\n",
      "Epoch 00020: val_loss improved from 3302.99318 to 3293.45098, saving model to ./model_window5hidden16_20dropout0.22batch200.h5\n",
      " - 10s - loss: 4080.8502 - mean_squared_error: 4080.8502 - mean_absolute_error: 48.0108 - val_loss: 3293.4510 - val_mean_squared_error: 3293.4510 - val_mean_absolute_error: 46.2851\n",
      "Epoch 21/30\n",
      "Epoch 00021: val_loss did not improve\n",
      " - 11s - loss: 4077.8651 - mean_squared_error: 4077.8651 - mean_absolute_error: 48.0267 - val_loss: 3387.2733 - val_mean_squared_error: 3387.2733 - val_mean_absolute_error: 47.5238\n",
      "Epoch 22/30\n",
      "Epoch 00022: val_loss did not improve\n",
      " - 10s - loss: 4061.0164 - mean_squared_error: 4061.0164 - mean_absolute_error: 47.9541 - val_loss: 3760.2049 - val_mean_squared_error: 3760.2049 - val_mean_absolute_error: 51.0497\n",
      "Epoch 23/30\n",
      "Epoch 00023: val_loss did not improve\n",
      " - 9s - loss: 4048.3199 - mean_squared_error: 4048.3199 - mean_absolute_error: 47.8463 - val_loss: 3402.6280 - val_mean_squared_error: 3402.6280 - val_mean_absolute_error: 48.0602\n",
      "Epoch 24/30\n",
      "Epoch 00024: val_loss improved from 3293.45098 to 3208.07822, saving model to ./model_window5hidden16_20dropout0.22batch200.h5\n",
      " - 11s - loss: 4040.2649 - mean_squared_error: 4040.2649 - mean_absolute_error: 47.7605 - val_loss: 3208.0782 - val_mean_squared_error: 3208.0782 - val_mean_absolute_error: 45.9213\n",
      "Epoch 25/30\n",
      "Epoch 00025: val_loss did not improve\n",
      " - 9s - loss: 4037.0065 - mean_squared_error: 4037.0065 - mean_absolute_error: 47.7739 - val_loss: 3620.6551 - val_mean_squared_error: 3620.6551 - val_mean_absolute_error: 49.7604\n",
      "Epoch 26/30\n",
      "Epoch 00026: val_loss did not improve\n",
      " - 10s - loss: 4020.2066 - mean_squared_error: 4020.2066 - mean_absolute_error: 47.6521 - val_loss: 3315.1013 - val_mean_squared_error: 3315.1013 - val_mean_absolute_error: 47.1795\n",
      "Epoch 27/30\n",
      "Epoch 00027: val_loss did not improve\n",
      " - 11s - loss: 4035.4189 - mean_squared_error: 4035.4189 - mean_absolute_error: 47.7063 - val_loss: 3535.4273 - val_mean_squared_error: 3535.4273 - val_mean_absolute_error: 49.5083\n",
      "Epoch 28/30\n",
      "Epoch 00028: val_loss did not improve\n",
      " - 11s - loss: 4005.1239 - mean_squared_error: 4005.1239 - mean_absolute_error: 47.5324 - val_loss: 3465.2887 - val_mean_squared_error: 3465.2887 - val_mean_absolute_error: 48.9044\n",
      "Epoch 29/30\n",
      "Epoch 00029: val_loss improved from 3208.07822 to 3148.58018, saving model to ./model_window5hidden16_20dropout0.22batch200.h5\n",
      " - 11s - loss: 4005.5608 - mean_squared_error: 4005.5608 - mean_absolute_error: 47.5320 - val_loss: 3148.5802 - val_mean_squared_error: 3148.5802 - val_mean_absolute_error: 45.8005\n",
      "Epoch 30/30\n",
      "Epoch 00030: val_loss did not improve\n",
      " - 11s - loss: 3983.5055 - mean_squared_error: 3983.5055 - mean_absolute_error: 47.3601 - val_loss: 3418.9356 - val_mean_squared_error: 3418.9356 - val_mean_absolute_error: 48.4840\n",
      "dict_keys(['val_mean_squared_error', 'mean_absolute_error', 'loss', 'val_loss', 'val_mean_absolute_error', 'mean_squared_error'])\n",
      "101362/101362 [==============================] - 3s 25us/step\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "lstm_7 (LSTM)                (None, 5, 32)             7552      \n",
      "_________________________________________________________________\n",
      "dropout_7 (Dropout)          (None, 5, 32)             0         \n",
      "_________________________________________________________________\n",
      "lstm_8 (LSTM)                (None, 20)                4240      \n",
      "_________________________________________________________________\n",
      "dropout_8 (Dropout)          (None, 20)                0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 1)                 21        \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 11,813\n",
      "Trainable params: 11,813\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 148973 samples, validate on 7841 samples\n",
      "Epoch 1/30\n",
      "Epoch 00001: val_loss improved from inf to 17635.31703, saving model to ./model_window5hidden32_20dropout0.5batch200.h5\n",
      " - 17s - loss: 18432.1852 - mean_squared_error: 18432.1852 - mean_absolute_error: 108.5427 - val_loss: 17635.3170 - val_mean_squared_error: 17635.3170 - val_mean_absolute_error: 106.7267\n",
      "Epoch 2/30\n",
      "Epoch 00002: val_loss improved from 17635.31703 to 14624.31375, saving model to ./model_window5hidden32_20dropout0.5batch200.h5\n",
      " - 13s - loss: 15302.4594 - mean_squared_error: 15302.4594 - mean_absolute_error: 95.7771 - val_loss: 14624.3137 - val_mean_squared_error: 14624.3137 - val_mean_absolute_error: 94.9963\n",
      "Epoch 3/30\n",
      "Epoch 00003: val_loss improved from 14624.31375 to 12098.01107, saving model to ./model_window5hidden32_20dropout0.5batch200.h5\n",
      " - 14s - loss: 12734.1419 - mean_squared_error: 12734.1419 - mean_absolute_error: 85.5456 - val_loss: 12098.0111 - val_mean_squared_error: 12098.0111 - val_mean_absolute_error: 85.3033\n",
      "Epoch 4/30\n",
      "Epoch 00004: val_loss improved from 12098.01107 to 10059.70777, saving model to ./model_window5hidden32_20dropout0.5batch200.h5\n",
      " - 14s - loss: 10689.7760 - mean_squared_error: 10689.7760 - mean_absolute_error: 77.6761 - val_loss: 10059.7078 - val_mean_squared_error: 10059.7078 - val_mean_absolute_error: 77.6603\n",
      "Epoch 5/30\n",
      "Epoch 00005: val_loss improved from 10059.70777 to 8509.11410, saving model to ./model_window5hidden32_20dropout0.5batch200.h5\n",
      " - 13s - loss: 9168.1176 - mean_squared_error: 9168.1176 - mean_absolute_error: 72.1459 - val_loss: 8509.1141 - val_mean_squared_error: 8509.1141 - val_mean_absolute_error: 72.0593\n",
      "Epoch 6/30\n",
      "Epoch 00006: val_loss improved from 8509.11410 to 7452.39651, saving model to ./model_window5hidden32_20dropout0.5batch200.h5\n",
      " - 14s - loss: 8132.8805 - mean_squared_error: 8132.8805 - mean_absolute_error: 68.7939 - val_loss: 7452.3965 - val_mean_squared_error: 7452.3965 - val_mean_absolute_error: 68.5057\n",
      "Epoch 7/30\n",
      "Epoch 00007: val_loss improved from 7452.39651 to 6194.72492, saving model to ./model_window5hidden32_20dropout0.5batch200.h5\n",
      " - 14s - loss: 7089.0752 - mean_squared_error: 7089.0752 - mean_absolute_error: 63.1224 - val_loss: 6194.7249 - val_mean_squared_error: 6194.7249 - val_mean_absolute_error: 61.7383\n",
      "Epoch 8/30\n",
      "Epoch 00008: val_loss improved from 6194.72492 to 5336.17228, saving model to ./model_window5hidden32_20dropout0.5batch200.h5\n",
      " - 12s - loss: 6116.0747 - mean_squared_error: 6116.0747 - mean_absolute_error: 57.2180 - val_loss: 5336.1723 - val_mean_squared_error: 5336.1723 - val_mean_absolute_error: 57.3654\n",
      "Epoch 9/30\n",
      "Epoch 00009: val_loss improved from 5336.17228 to 4838.08793, saving model to ./model_window5hidden32_20dropout0.5batch200.h5\n",
      " - 13s - loss: 5545.0578 - mean_squared_error: 5545.0578 - mean_absolute_error: 54.4196 - val_loss: 4838.0879 - val_mean_squared_error: 4838.0879 - val_mean_absolute_error: 55.3034\n",
      "Epoch 10/30\n",
      "Epoch 00010: val_loss improved from 4838.08793 to 4324.18363, saving model to ./model_window5hidden32_20dropout0.5batch200.h5\n",
      " - 13s - loss: 5176.9823 - mean_squared_error: 5176.9823 - mean_absolute_error: 52.8811 - val_loss: 4324.1836 - val_mean_squared_error: 4324.1836 - val_mean_absolute_error: 52.5189\n",
      "Epoch 11/30\n",
      "Epoch 00011: val_loss improved from 4324.18363 to 3977.09921, saving model to ./model_window5hidden32_20dropout0.5batch200.h5\n",
      " - 13s - loss: 4955.7832 - mean_squared_error: 4955.7832 - mean_absolute_error: 52.0162 - val_loss: 3977.0992 - val_mean_squared_error: 3977.0992 - val_mean_absolute_error: 50.0543\n",
      "Epoch 12/30\n",
      "Epoch 00012: val_loss improved from 3977.09921 to 3767.55455, saving model to ./model_window5hidden32_20dropout0.5batch200.h5\n",
      " - 13s - loss: 4812.3493 - mean_squared_error: 4812.3493 - mean_absolute_error: 51.4644 - val_loss: 3767.5545 - val_mean_squared_error: 3767.5545 - val_mean_absolute_error: 49.1643\n",
      "Epoch 13/30\n",
      "Epoch 00013: val_loss improved from 3767.55455 to 3467.40491, saving model to ./model_window5hidden32_20dropout0.5batch200.h5\n",
      " - 13s - loss: 4729.0726 - mean_squared_error: 4729.0726 - mean_absolute_error: 51.1602 - val_loss: 3467.4049 - val_mean_squared_error: 3467.4049 - val_mean_absolute_error: 46.8466\n",
      "Epoch 14/30\n",
      "Epoch 00014: val_loss did not improve\n",
      " - 12s - loss: 4694.2134 - mean_squared_error: 4694.2134 - mean_absolute_error: 51.0649 - val_loss: 3481.1414 - val_mean_squared_error: 3481.1414 - val_mean_absolute_error: 47.3835\n",
      "Epoch 15/30\n",
      "Epoch 00015: val_loss did not improve\n",
      " - 12s - loss: 4652.4308 - mean_squared_error: 4652.4308 - mean_absolute_error: 50.9261 - val_loss: 3498.9110 - val_mean_squared_error: 3498.9110 - val_mean_absolute_error: 47.6375\n",
      "Epoch 16/30\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-35-440add01784f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     43\u001b[0m                 history1 = model.fit(train_features, train_labels, epochs=30, batch_size=200, validation_split=0.05, verbose=2,\n\u001b[0;32m     44\u001b[0m                           callbacks = [EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=2, mode='min'),\n\u001b[1;32m---> 45\u001b[1;33m                                        ModelCheckpoint(model_path,monitor='val_loss', save_best_only=True, mode='min', verbose=2)]\n\u001b[0m\u001b[0;32m     46\u001b[0m                           )\n\u001b[0;32m     47\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\BCI-EXPERT\\Anaconda3\\lib\\site-packages\\keras\\models.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[0;32m    958\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    959\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 960\u001b[1;33m                               validation_steps=validation_steps)\n\u001b[0m\u001b[0;32m    961\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    962\u001b[0m     def evaluate(self, x, y, batch_size=32, verbose=1,\n",
      "\u001b[1;32mC:\\Users\\BCI-EXPERT\\Anaconda3\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[0;32m   1655\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1656\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1657\u001b[1;33m                               validation_steps=validation_steps)\n\u001b[0m\u001b[0;32m   1658\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1659\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[1;32mC:\\Users\\BCI-EXPERT\\Anaconda3\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[1;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[0;32m   1211\u001b[0m                     \u001b[0mbatch_logs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'size'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1212\u001b[0m                     \u001b[0mcallbacks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1213\u001b[1;33m                     \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1214\u001b[0m                     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1215\u001b[0m                         \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\BCI-EXPERT\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2355\u001b[0m         \u001b[0msession\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2356\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[1;32m-> 2357\u001b[1;33m                               **self.session_kwargs)\n\u001b[0m\u001b[0;32m   2358\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2359\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\BCI-EXPERT\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    787\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    788\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 789\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    790\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    791\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\BCI-EXPERT\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    995\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    996\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m--> 997\u001b[1;33m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[0;32m    998\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    999\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\BCI-EXPERT\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1130\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1131\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[1;32m-> 1132\u001b[1;33m                            target_list, options, run_metadata)\n\u001b[0m\u001b[0;32m   1133\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1134\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[1;32mC:\\Users\\BCI-EXPERT\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1137\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1138\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1139\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1140\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1141\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\BCI-EXPERT\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1119\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[0;32m   1120\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1121\u001b[1;33m                                  status, run_metadata)\n\u001b[0m\u001b[0;32m   1122\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1123\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "\n",
    "#################################################3 MACHINE LEARNING ###########################################\n",
    "\n",
    "import pickle\n",
    "best_cv_mse = 10**20\n",
    "modelidx=1\n",
    "\n",
    "''' KICKSTART INTO TRAINING'''\n",
    "\n",
    "units2 = 20\n",
    "for windowlength in [5,18]:   \n",
    "    for dropout in [0.22,0.5]:\n",
    "        for units1 in [32, 16]:   \n",
    "                filename = 'datasets_final' + 'windowlen' + str(windowlength)\n",
    "                with open(filename, 'rb') as f:\n",
    "                    [train_features,train_labels,test_features,test_labels,cv_features,cv_labels] = pickle.load(f)\n",
    "            #                  idx = np.random.randint(100, size=2)\n",
    "            #                  train_s = train_features[idx,:]\n",
    "            #                  train_l_s = train_labels[idx,:]\n",
    "                featurecount = 26\n",
    "                model = Sequential()\n",
    "                model.add(LSTM(\n",
    "                         input_shape=(windowlength, featurecount),\n",
    "                         units=units1, # number of hidden units in the 1st hidden layer -- use 32-multiple\n",
    "                         return_sequences=True))\n",
    "                model.add(Dropout(dropout))\n",
    "                model.add(LSTM(\n",
    "                          units=units2, # number of hidden units in the 2nd hidden layer\n",
    "                          return_sequences=False))\n",
    "                model.add(Dropout(dropout))\n",
    "                 # ''' OPTIMIZE DROPOUT '''\n",
    "                \n",
    "                model.add(Dense(units=1)) # add dense output layer\n",
    "                model.add(Activation(\"linear\")) # '''******** CHOOSE ACTIVATION FUN ********\n",
    "                model.compile(loss='mean_squared_error', optimizer='rmsprop',metrics=['mse','mae'])\n",
    "                \n",
    "                print(model.summary())\n",
    "                model_path = './model_window'+str(windowlength)+'hidden'+str(units1)+'_'+str(units2)+'dropout'+str(dropout)+'batch'+str(200)+'.h5'\n",
    "                \n",
    "                 \n",
    "                history1 = model.fit(train_features, train_labels, epochs=30, batch_size=200, validation_split=0.05, verbose=2,\n",
    "                          callbacks = [EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=2, mode='min'),\n",
    "                                       ModelCheckpoint(model_path,monitor='val_loss', save_best_only=True, mode='min', verbose=2)]\n",
    "                          )\n",
    "                \n",
    "                # list all data in history\n",
    "                print(history1.history.keys())\n",
    "                \n",
    "                cv_scores = model.evaluate(cv_features,cv_labels, batch_size=200,verbose=1)\n",
    "                cv_mse = cv_scores[1]\n",
    "                if cv_mse < best_cv_mse:\n",
    "                    best_model = model\n",
    "                    best_cv_mse = cv_mse \n",
    "                modelidx +=1            \n",
    "\n",
    "test_scores = best_model.evaluate(test_features,test_labels, batch_size=200,verbose=1)\n",
    "print('\\nBest model MSE: {}'.format(test_scores[1]))\n",
    "print('\\nBest model MAE: {}'.format(test_scores[2]))\n",
    "\n",
    "test_predictions = best_model.predict(test_features)\n",
    "test_predictions\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bayesian Optimization using Hyperas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following routine seemed to run very slowly, so it was not practical to wait around that much  on a slow laptop. Parallel Training on Mongo workers is indicated. Further advice very welcome!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.datasets import mnist\n",
    "from keras.layers.core import Dense, Dropout, Activation\n",
    "from keras.models import Sequential\n",
    "from keras.utils import np_utils\n",
    "from keras.preprocessing import sequence\n",
    "from keras.datasets import imdb\n",
    "from keras.models import Sequential\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.recurrent import LSTM\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from keras.callbacks import TensorBoard\n",
    "\n",
    "from hyperas import optim\n",
    "from hyperas.distributions import choice, uniform, conditional\n",
    "\n",
    "\n",
    "\n",
    "from hyperopt import fmin, tpe, hp, STATUS_OK, Trials\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import sys\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def create_data(X,y,X_val,y_val):\n",
    "        return X,y,X_val,y_val\n",
    "\n",
    "X,y,X_val,y_val = create_data(X,y,X_val,y_val)\n",
    "\n",
    "def create_model(X, X_val, y, y_val):\n",
    "    from keras.models import Sequential\n",
    "    from keras.layers.core import Dense, Dropout, Activation\n",
    "    from keras.optimizers import Adadelta, Adam, rmsprop\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def get_space():\n",
    "    return {\n",
    "    'LSTM': choice('LSTM', [32, 64, 128]),\n",
    "    'Dropout': uniform('Dropout', 0, 1),\n",
    "    'optimizer': choice('optimizer', ['rmsprop', 'adam', 'sgd']),\n",
    "    'batch_size': choice('batch_size', [64, 128]),\n",
    "    'nb_epoch': choice('nb_epoch', [10, 20]),\n",
    "    }\n",
    "\n",
    "\n",
    "########################### VER 1 ##################################\n",
    "space = {   'units1': hp.choice('units1', [32,64]),\n",
    "            'units2': hp.choice('units2', [13,26]),\n",
    "\n",
    "            'dropout1': hp.uniform('dropout1', .25,.75),\n",
    "            'dropout2': hp.uniform('dropout2',  .25,.75),\n",
    "\n",
    "            'batch_size' : hp.choice('batch_size', [32,64,128]),\n",
    "\n",
    "            'epochs' :  5,\n",
    "            'optimizer': hp.choice('optimizer',['adadelta','adam','rmsprop']),\n",
    "            'activation': hp.choice('activation',['linear','relu'])\n",
    "        }\n",
    "\n",
    "#'choice': hp.choice('num_layers',\n",
    "#                    [ {'layers':'two', },\n",
    "#                    {'layers':'three',\n",
    "#                    'units3': hp.choice('units3', [2]), \n",
    "#                    'dropout3': hp.uniform('dropout3', .25,.75)}\n",
    "#                    ]),\n",
    "\n",
    "def f_nn(params):   \n",
    "    from keras.models import Sequential\n",
    "    from keras.layers.core import Dense, Dropout, Activation\n",
    "    from keras.optimizers import Adadelta, Adam, rmsprop\n",
    "\n",
    "    print ('Params testing: ', params)\n",
    "    model = Sequential()\n",
    "    #model.add(Dense(output_dim=params['units1'], input_dim = X.shape[1])) \n",
    "    #model.add(Activation(params['activation']))\n",
    "    model.add(LSTM(\n",
    "         input_shape=(32, 29),\n",
    "         units=params['units1'], \n",
    "         return_sequences=True))\n",
    "    model.add(Dropout(params['dropout1']))\n",
    "    model.add(LSTM(units=params['units2'],return_sequences=False))\n",
    "    model.add(Dropout(params['dropout2']))\n",
    "    model.add(Dense(units = 1))\n",
    "    model.add(Activation('linear'))\n",
    "    model.compile(loss='mean_squared_error', optimizer=params['optimizer'])\n",
    "    model.fit(X, y, epochs=params['epochs'], batch_size=params['batch_size'], verbose = 2)\n",
    "    score, acc = model.evaluate(X_val, y_val, show_accuracy=True, verbose=2)\n",
    "\n",
    "    return {'loss': -acc, 'status': STATUS_OK}\n",
    "\n",
    "# ,'model':model\n",
    "trials = Trials()\n",
    "best = fmin(f_nn, space, algo=tpe.suggest, max_evals=3, trials=trials)\n",
    "\n",
    "\n",
    "\n",
    "########################################### REWRITING TO BE MORE CLEAN ###################################\n",
    "\n",
    "def f_nn(params):   \n",
    "    from keras.models import Sequential\n",
    "    from keras.layers.core import Dense, Dropout, Activation\n",
    "    from keras.optimizers import Adadelta, Adam, rmsprop\n",
    "\n",
    "    print ('Params testing: ', params)\n",
    "    model = Sequential()\n",
    "    #model.add(Dense(output_dim=params['units1'], input_dim = X.shape[1])) \n",
    "    #model.add(Activation(params['activation']))\n",
    "    model.add(LSTM(\n",
    "         input_shape=(32, 29),\n",
    "         units=params['units1'], \n",
    "         return_sequences=True, dropout = params['dropout1']))\n",
    "    model.add(LSTM(units=params['units2'],return_sequences=False,dropout = params['dropout2']))\n",
    "    model.add(Dense(units = 1,activation='linear'))\n",
    "    model.compile(loss='mean_squared_error', optimizer=params['optimizer'])\n",
    "\n",
    "    model.fit(X, y, epochs=params['epochs'], batch_size=params['batch_size'], verbose = 2)\n",
    "    score, acc = model.evaluate(X_val, y_val, show_accuracy=True, verbose=2)\n",
    "\n",
    "    return {'loss': -acc, 'status': STATUS_OK, 'model':model}\n",
    "###############################################\n",
    "    \n",
    "    \n",
    "    \n",
    "    ############################################### VER 2 ############################################33\n",
    "    \n",
    "space2 = {\n",
    "            'dropout1': hp.uniform('dropout1', .25,.75),\n",
    "        }\n",
    "        \n",
    "def f2_nn(params):   \n",
    "    from keras.models import Sequential\n",
    "    from keras.layers.core import Dense, Dropout, Activation\n",
    "    from keras.optimizers import Adadelta, Adam, rmsprop\n",
    "\n",
    "    print ('Params testing: ', params)\n",
    "    model = Sequential()\n",
    "    #model.add(Dense(output_dim=params['units1'], input_dim = X.shape[1])) \n",
    "    #model.add(Activation(params['activation']))\n",
    "    model.add(LSTM(\n",
    "         input_shape=(32, 29),\n",
    "         units=20, \n",
    "         return_sequences=True, dropout = params['dropout1']))\n",
    "    model.add(LSTM(units=10,return_sequences=False,dropout = 0.2))\n",
    "    model.add(Dense(units = 1,activation='linear'))\n",
    "    model.compile(loss='mean_squared_error', optimizer='rmsprop')\n",
    "\n",
    "    model.fit(X, y, epochs=5, batch_size=32, verbose = 2)\n",
    "    score, acc = model.evaluate(X_val, y_val, show_accuracy=True, verbose=2)\n",
    "    return {'loss': -acc, 'status': STATUS_OK, 'model':model}\n",
    "    \n",
    "best2 = fmin(f2_nn, space2, algo=tpe.suggest, max_evals=3, trials=trials)\n",
    "# Info on why fmin : https://github.com/hyperopt/hyperopt/wiki/FMin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "101362/101362 [==============================] - 188s 2ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[206.89023 ],\n",
       "       [205.98921 ],\n",
       "       [205.09381 ],\n",
       "       ...,\n",
       "       [ 75.5407  ],\n",
       "       [ 80.448456],\n",
       "       [ 68.66181 ]], dtype=float32)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_predictions = best_model.predict(test_features,verbose=1,batch_size=1)\n",
    "test_predictions"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Anaconda3]",
   "language": "python",
   "name": "Python [Anaconda3]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
